Proyecto final - NLP
Electiva NLP
Fabian Camilo Cabrera
Alejandro Fandiño
In [1]:
import re
from collections import Counter
import pandas as pd
import numpy as np
import urllib
import seaborn as sns
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
import plotly.io as pio
import plotly.express as px
import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
In [2]:
auspol = pd.read_csv("auspol2019.csv")
auspol.head()
Out[2]:
In [3]:
location = pd.read_csv("location_geocode.csv")
location.head()
Out[3]:
In [4]:
data=auspol.merge(location,how='inner',left_on='user_location',right_on='name')
data.head()
Out[4]:
In [5]:
data.isnull().sum()
Out[5]:
In [6]:
data=data.dropna()
In [7]:
df = data[['created_at','full_text','retweet_count','favorite_count','user_name','user_description','user_location','lat','long']]
df = df.copy()
df.head()
Out[7]:
In [8]:
print('Dimensión de la data:', df.shape)
print('Nombre de las columnas:', '\n' ,df.columns)
In [9]:
print('Máximo de retweets:', df.retweet_count.max())
print('Maximo de tweets favoritos:', df.favorite_count.max())
In [10]:
print('Fecha y hora de creación:', df.created_at.min())
print('Fecha y hora de último registro:', df.created_at.max())
Visualizaciones¶
Serie de tiempo¶
In [11]:
df['created_at'] = pd.to_datetime(df['created_at'])
fecha = df['created_at'].dt.date.value_counts()
fecha = fecha.sort_index()
In [12]:
fecha = fecha.to_frame().reset_index().rename(columns={'index':'date','created_at':'count'})
fecha.head()
Out[12]:
In [13]:
fig=go.Figure(go.Scatter(x=fecha['date'],
y=fecha['count'],
mode='markers+lines',
name="Submissions",
marker_color='dodgerblue'),
)
fig.update_layout(
title_text='Tweets por día : ({} - {}) '.format(fecha['date'].sort_values()[0].strftime("%d/%m/%Y"),
fecha['date'].sort_values().iloc[-1].strftime("%d/%m/%Y")),template="plotly_white",
title_x=0.5,height=450)
fig.show()
De lo anterior, vemos que durante diez días se recopilaron los tweets entre los dias 10 y 20 de mayo de 2019. Vemos que hay un pico alto en el día 18 de mayo que coincide con la fecha de las elecciones oficiales en Australia.
In [14]:
dia=df.loc[(df['created_at'].dt.day == 18), 'created_at']
horas=dia.dt.hour.value_counts()
horas=horas.sort_index()
horas=horas.to_frame().reset_index().rename(columns={'index':'hora','created_at':'cnt'})
horas.head()
Out[14]:
In [15]:
fig = go.Figure(go.Scatter(x=horas['hora'],
y=horas['cnt'],
mode='markers+lines',
marker_color=horas['cnt'])) # hover text goes here
fig.update_layout(
title_text='Horas del día : ({} - {}) horas '.format(horas['hora'].sort_values()[0],
horas['hora'].sort_values().iloc[-1]),template="plotly_white",
title_x=0.5, height=500)
fig.show()
Para el día de las elecciones (18 de mayo) encontramos que entre las 8am hasta las 3pm hubo un flujo mayor de partición en twitter, ya que coincide con la hora de apertura y cierre de las elecciones en Australia.
Número de tweets por usuario¶
In [17]:
plt.figure(figsize=(14,7))
usuarios = df['user_name'].value_counts()
usuarios = usuarios[:15,]
sns.barplot(usuarios.values, usuarios.index,data=df)
plt.title("Top 10 de los números de tweets por usuario")
plt.xticks(rotation='horizontal')
plt.show()
Hastag¶
In [18]:
def encontrar_hash(text):
line=re.findall(r'(?<=#)\w+',text)
return " ".join(line)
df['hash']=df['full_text'].apply(lambda x:encontrar_hash(x))
In [19]:
hastags=list(df[(df['hash'].notnull())&(df['hash']!="")]['hash'])
hastags = [each_string.lower() for each_string in hastags]
hash_df=dict(Counter(hastags))
top_hash_df=pd.DataFrame(list(hash_df.items()),columns = ['word','count']).sort_values('count',ascending=False)[:10]
top_hash_df
Out[19]:
In [20]:
fig = go.Figure(go.Bar(
x=top_hash_df['word'],y=top_hash_df['count'],
marker={'color': top_hash_df['count'],
'colorscale': 'reds'},
text=top_hash_df['count'],
textposition = "outside",
))
fig.update_layout(title_text='Los 10 hastags más comunes',xaxis_title="Hashtags",
yaxis_title="frecuencia",template="plotly_white",height=700,title_x=0.5)
fig.show()
Menciones¶
In [21]:
def mentions(text):
line=re.findall(r'(?<=@)\w+',text)
return " ".join(line)
In [22]:
df['mentions']=df['full_text'].apply(lambda x:mentions(x))
temp=df['mentions'].value_counts()[:][1:11]
temp =temp.to_frame().reset_index().rename(columns={'index':'Mentions','mentions':'count'})
temp.head()
Out[22]:
In [23]:
fig = go.Figure(go.Bar(
x=temp['Mentions'],y=temp['count'],
marker={'color': temp['count'],
'colorscale': 'greens'},
text=temp['count'],
textposition = "outside",
))
fig.update_layout(title_text='Las 10 menciones más comunes',xaxis_title="Mentions",
yaxis_title="frecuencia",template="plotly_white",height=700,title_x=0.5)
fig.show()
Emojis¶
In [24]:
import regex
import emoji
def get_emojis(text):
emoji_list = []
data = regex.findall(r'\X', text)
for word in data:
if any(char in emoji.UNICODE_EMOJI for char in word):
emoji_list.append(word)
return emoji_list
df['emojis'] = df['full_text'].apply(lambda text: get_emojis(text))
In [25]:
from collections import Counter
temp_emojis = pd.DataFrame(list(zip(Counter(sum(df.emojis.values,[])).keys(),Counter(sum(df.emojis.values,[])).values())))
temp_emojis.columns = ['emoji', 'cnt']
temp_emojis.sort_values('cnt', ascending=False, inplace=True)
temp_emojis.head()
Out[25]:
In [26]:
trace = go.Bar(x=temp_emojis.emoji.values[:10],
y=temp_emojis.cnt.values[:10])
layout = go.Layout(title='Emojis más usados')
fig = go.Figure(data=[trace], layout=layout)
iplot(fig)
Palabras más comunes¶
In [36]:
from nltk.corpus import stopwords
stopwords_sp = stopwords.words('english')
def pre_procesado1(texto):
texto = texto.lower()
texto = re.sub(r"[\W\d_]+", " ", texto)
texto = [palabra for palabra in texto.split() if palabra not in stopwords_sp]
texto = " ".join(texto)
return texto
df['pp'] = df['clean'].apply(lambda texto: pre_procesado1(texto))
from sklearn.feature_extraction.text import CountVectorizer
vec = CountVectorizer().fit(df['pp'])
bag_of_words = vec.transform(df['pp'])
sum_words = bag_of_words.sum(axis=0)
words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
words_freq
Out[36]:
Nube de palabras¶
In [24]:
import re
pattern1 = '?P<pic>pic.twitter.com/[^\s]+'
pattern2 = '?P<url>https?://[^\s]+'
def text_clean(row):
text = row['full_text']
links = [tuple(j for j in i if j)[-1] for i in re.findall(f"({pattern1})|({pattern2})",text)]
for link in links:
text = text.replace(link,"")
hashtags = [interaction for interaction in text.split() if interaction.startswith("#")]
for hashtag in hashtags:
text = text.replace(hashtag,"")
mentions = [interaction for interaction in text.split() if interaction.startswith("@")]
for mention in mentions:
text = text.replace(mention,"")
return text, links, hashtags, mentions
In [27]:
df[['clean', 'links', 'hashtags', 'mentions']] = df.apply(text_clean, axis=1, result_type='expand')
In [29]:
import PIL.Image
australia = PIL.Image.open('australiaMapa.png')
#display(australia)
In [30]:
mask = np.array(australia)
In [31]:
palabras = df.clean.str.cat(sep=" ")
wordcloud = WordCloud(width=800, height=400,
max_font_size=150, max_words=250,
background_color='white', colormap='copper_r',
stopwords=stopwords_sp,
mask=mask, contour_width=0.5, contour_color='green').generate(palabras) # https://matplotlib.org/3.1.0/tutorials/colors/colormaps.html
#wordcloud.to_file("australia.jpg")
plt.figure(figsize=(10,8))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()
Se puede ver que hay al menos dos partidos en Australia: "Laborista" y "Liberal". Uno de estos dos partidos está dirigido por Scott Morrison, otro puede estar dirigido por Bill Shorten.
Mapa de calor¶
In [32]:
import re
from nltk.corpus import stopwords
stopwords_sp = stopwords.words('english')
from sklearn.feature_extraction.text import TfidfVectorizer
def pre_procesado(texto):
texto = texto.lower()
texto = re.sub(r"[\W\d_]+", " ", texto)
texto = " ".join([palabra for palabra in texto.split() if palabra not in stopwords_sp])
return texto
tfidf_vect = TfidfVectorizer(preprocessor=pre_procesado, ngram_range=(1,1), max_features=100)
tfidf = tfidf_vect.fit_transform(df.clean.values)
In [33]:
from sklearn.metrics.pairwise import cosine_similarity
voc = [k for k,v in sorted(tfidf_vect.vocabulary_.items(), key=lambda kv: kv[1])]
temp = pd.DataFrame(tfidf.toarray())
temp = cosine_similarity(temp.T.values)
temp = pd.DataFrame(temp)
temp.columns = voc
temp.index = voc
temp
Out[33]:
In [34]:
import scipy.cluster.hierarchy as sch
import numpy as np
pairwise_distances = sch.distance.pdist(temp)
linkage = sch.linkage(pairwise_distances, method='ward')
idx_to_cluster_array = sch.fcluster(linkage, pairwise_distances.max() * 1, criterion='distance')
idx = np.argsort(idx_to_cluster_array)
temp = temp.copy()
temp2 = temp.iloc[idx, :].T.iloc[idx, :]
my_idx = idx_to_cluster_array
In [35]:
trace = go.Heatmap(z=temp2.values.tolist(),
x=temp2.index.values,
y=temp2.columns.values,
colorscale='Oranges')
layout = go.Layout(title='🔥 Mapa de calor entre palabras (ordenado)🔥',
width=800, height=800)
fig = go.Figure(data=[trace],layout=layout)
iplot(fig)
Ubicación de los usuarios¶
In [47]:
localizacion = df[['lat','long']]
localizacion.columns = localizacion.columns.str.strip()
In [48]:
import folium
from folium import plugins
from folium.plugins import HeatMap
In [49]:
mapa = folium.Map(location=[-25.274398,133.775136],zoom_start = 5)
In [50]:
heat_df = localizacion[['lat', 'long']]
heat_df = heat_df.dropna(axis=0, subset=['lat','long'])
# List comprehension para hacer una lista de listas
heat_data = [[row['lat'],row['long']] for index, row in heat_df.iterrows()]
HeatMap(heat_data).add_to(mapa)
mapa
Out[50]:
Gráfico de puntos¶
In [53]:
map_osm = folium.Map(location=[-25.274398,133.775136],zoom_start=15)
In [54]:
subset_of_df = localizacion.sample(n=1000,random_state=1)
In [55]:
some_map = folium.Map(location=[ subset_of_df['lat'].mean(),
subset_of_df['long'].mean()],
zoom_start=10)
for row in subset_of_df.itertuples():
some_map.add_child(folium.Marker(location=[row.lat,row.long]))
In [56]:
some_map
Out[56]:
Análisis de sentimiento¶
In [27]:
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
vader = SentimentIntensityAnalyzer()
df['sentimiento'] = df['full_text'].apply(lambda valor: vader.polarity_scores(valor)['compound'])
df.head()
Out[27]:
In [28]:
cut = pd.cut(df['sentimiento'],[-np.inf, -.01, .01, np.inf],labels=['negativo', 'neutro', 'positivo'])
df['polaridad'] = cut.values
df[['polaridad','sentimiento']].head()
Out[28]:
In [29]:
df[['full_text', 'sentimiento','polaridad']].sort_values('sentimiento')
Out[29]:
In [30]:
print('Número de tweets negativos:' ,len(df.sentimiento.loc[df.sentimiento<0]))
In [31]:
print('Número de tweets positivos:' ,len(df.sentimiento.loc[df.sentimiento>0]))
In [32]:
print('Número de tweets neutros:' ,len(df.sentimiento.loc[df.sentimiento==0]))
In [33]:
plt.figure(figsize=(14, 6))
polaridad = df['polaridad'].value_counts()
splot=sns.barplot(x=polaridad.index,y=polaridad.values,data=df)
for p in splot.patches:
splot.annotate(format(p.get_height(), '.1f'),
(p.get_x() + p.get_width() / 2., p.get_height()),
ha = 'center', va = 'center',
xytext = (0, 9),
textcoords = 'offset points')
plt.ylabel("Frecuencia", size=14)
Out[33]:
In [47]:
labels = df.polaridad.value_counts().index
colors = ['blue','darkorange','yellow']
sizes = df.polaridad.value_counts().values
plt.figure(0,figsize = (7,7))
plt.pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%')
plt.title('Sentimientos',color = 'blue',fontsize = 15)
plt.show()
Modelo LDA¶
In [58]:
import pyLDAvis.gensim
from gensim.models import LdaModel
from gensim.corpora import Dictionary
from pprint import pprint
In [59]:
from nltk.corpus import stopwords
stopwords_sp = stopwords.words('english')
def pre_procesado1(texto):
texto = texto.lower()
texto = re.sub(r"[\W\d_]+", " ", texto)
texto = [palabra for palabra in texto.split() if palabra not in stopwords_sp]
return texto
In [60]:
df['pre_procesado'] = df['full_text'].apply(lambda texto: pre_procesado1(texto))
In [61]:
df.head()
Out[61]:
In [62]:
# Crear una representación de los documentos en forma de diccionario
dictionary = Dictionary(df['pre_procesado'].values)
In [63]:
# Filtrar palabras muy frecuentes o infrecuentes
dictionary.filter_extremes(no_below=5, no_above=0.5)
corpus = [dictionary.doc2bow(text) for text in df['pre_procesado'].values]
In [64]:
# Train the topic model
model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=7, passes=4)
In [45]:
model.print_topics(num_words=10)
Out[45]:
In [65]:
import pyLDAvis.gensim
lda_display = pyLDAvis.gensim.prepare(model, corpus, dictionary, sort_topics=True)
pyLDAvis.display(lda_display)
Out[65]:
El primer topico vemos que hay una gran cantidad de palabras en la que resalta 'labor' del partido laborista, por otro lado, el topico tres y cuatro tienen palabras comunes y las que se destacan son: 'morrison', 'labor' y 'shorten', esto podria indicar como se dividen los documentos entre los dos candidatos y tambien vemos la palabra 'win' dando una tendencia de ganador a Morrison.
In [66]:
d = dictionary.doc2bow(["shorten", "morrinson"])
topics = model.get_document_topics(d)
topics
Out[66]:
In [59]:
def get_doc_top_n(text_processed, n):
d = dictionary.doc2bow(text_processed)
topics = dict(model.get_document_topics(d))
try:
return topic[n]
except:
return None
get_doc_top_n(["election", "shorten", "morrinson", "australia"],1)
In [60]:
for t in range(0,7):
top_name = f"topic_{t}"
df[top_name] = df['pre_procesado'].apply(lambda doc: get_doc_top_n(doc,t))
In [61]:
for t in range(0,7):
print(f"****************************** TOPIC {t} ******************************")
for i,row in enumerate(df.sort_values(f"topic_{t}", ascending=False)['full_text'].head()):
print(f"tweet #{i+1}")
print(row[:500])
print()
print()
k-means¶
In [10]:
df_muestra = df.sample(n=500, random_state=1)
In [11]:
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
def pre_procesado2(texto):
texto = texto.lower()
texto = re.sub(r"[\W\d_]+", " ", texto)
return texto
tfidf_vect = TfidfVectorizer(preprocessor=pre_procesado2)
tfidf = tfidf_vect.fit_transform(df_muestra.full_text.values)
tfidf_matrix = pd.DataFrame(tfidf.toarray(), columns=tfidf_vect.get_feature_names())
In [12]:
# Escogiendo el K nodo
from sklearn.neighbors import NearestNeighbors
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
In [13]:
ks = []
kinertia = []
for k in range(1,10):
kmeans = KMeans(n_clusters=k).fit(tfidf_matrix)
ks.append(k)
kinertia.append(kmeans.inertia_)
plt.xlabel("Número de cúmulos");
plt.ylabel("Inercia");
plt.title('Método del codo');
plt.plot(ks,kinertia, 'bx-');
In [19]:
class color:
PURPLE = '\033[95m'
CYAN = '\033[96m'
DARKCYAN = '\033[36m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
END = '\033[0m'
k_pt = 4
model = KMeans(n_clusters=k_pt)
model.fit(tfidf_matrix)
nbrs = NearestNeighbors(n_neighbors=3, metric='euclidean').fit(tfidf_matrix.values)
df_muestra['cluster'] = model.labels_
clust_cnt = df_muestra['cluster'].value_counts()
clust_cnt_pct = df_muestra['cluster'].value_counts(normalize=True)
print(f"{color.BOLD}CLUSTERS:\n{color.END}")
centroids = model.cluster_centers_
order_centroids = model.cluster_centers_.argsort()[:, ::-1]
terms = tfidf_vect.get_feature_names()
for i in range(k_pt):
print(f"{color.BLUE}Cluster {i}:{color.END}")
print(f"{color.CYAN}COUNT {color.END} {clust_cnt[i]} comments ({clust_cnt_pct[i]:.2%} of the data)")
print(f"{color.CYAN}TERMS {color.END}", end=" ")
for ind in order_centroids[i, :20]:
print(f'{color.BOLD}{terms[ind]}{color.END}', end=" "),
print(f"\n{color.CYAN}REPRESENTATIVE COMMENTS{color.END}")
for comment in df_muestra.iloc[nbrs.kneighbors([centroids[i]])[1][0]]['full_text'].values:
print(f"* {comment}")
print("\n")
In [20]:
clusters = {0:'1',
1:'2',
2:'3',
3:'4'
}
df_muestra['cluster'] = df_muestra['cluster'].apply(lambda val: clusters[val])
df_muestra.sample(2)
Out[20]:
In [21]:
df_centroides = pd.DataFrame(model.cluster_centers_)
df_centroides['cluster'] = clusters.values()
df_centroides
Out[21]:
In [22]:
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import plotly.graph_objs as go
from plotly.offline import iplot
In [24]:
pca = PCA(n_components=2)
result = pca.fit_transform(tfidf_matrix)
result = pd.DataFrame(result)
result.columns = ['X', 'Y']
result['cluster'] = df_muestra.cluster.values
result['texto'] = df_muestra.full_text.apply(lambda val: val[:140])
colorsIdx = {'1': 'blue',
'2': 'yellow',
'3': 'red',
'4': 'green'
}
cols = df_muestra['cluster'].map(colorsIdx)
trace = go.Scatter(x=result['X'].values,
y=result['Y'].values,
text=result['texto'].values,
mode='markers',
marker=dict(color=cols))
layout = go.Layout(title="PCA")
fig = go.Figure(data=trace, layout=layout)
iplot(fig)
Vemos para este caso que el K-means no es el mejor algoritmo para aplicar sobre estos datos porque están muy relacionados los clusters el uno con el otro, por esta razón es dificil ver claramente cuales son los cluster.